{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "couldn't import doomish\n",
      "Couldn't import doom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([-0.03846154,  0.        , -0.53846154, -0.62222222])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "import gym_pygame\n",
    "\n",
    "#!pip install git+https://github.com/ntasfi/PyGame-Learning-Environment.git\n",
    "#!pip install git+https://github.com/qlan3/gym-games.git\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Catcher-PLE-v0')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box(-inf, inf, (4,), float32)\n",
      "env.action_space= Discrete(3)\n",
      "state= [-0.03846154  0.         -0.07692308 -0.8       ]\n",
      "action= 1\n",
      "next_state= [ 0.          0.1512     -0.07692308 -0.73333333]\n",
      "reward= 0.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
      "  logger.deprecation(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
      "  logger.warn(f\"{pre} is not within the observation space.\")\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:272: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEWCAYAAACjTbhPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVe0lEQVR4nO3df2xVd/3H8Ve73nv5Ue5tKXIvlV7AiKsTmVsZ5YpmidxI5qIDmskfM6IuLsAFgfnH0j9gkqiXbDEqBoc/Ipg4V60JU5ZMJGV02XJXaBcUBnbgSHpjey8ues8tjP4I9/P9w68nu6Ol98IH7i08H8k7WT/nc895c5a+cu45vfdTYYwxAgCLKkvdAIDbD8ECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwLqbFix79uzR/PnzNWXKFDU3N+vYsWM361AAykzFzfis0O9+9zt99atf1d69e9Xc3Kwf/ehHam9vV29vr2bPnn3N1+ZyOfX392vGjBmqqKiw3RqA62SM0eDgoOrr61VZOcE1ibkJli5damKxmPvzlStXTH19vYnH4xO+NplMGkkURZVpJZPJCX+Prb8VGhkZUU9Pj6LRqDtWWVmpaDSqRCJx1fzh4WFls1m3DB+2BsrajBkzJpxjPVjeffddXblyRcFgMG88GAwqlUpdNT8ejysQCLgVDodttwTAokJuUZT8qVBra6scx3ErmUyWuiUAN6jK9g5nzZqlu+66S+l0Om88nU4rFApdNd/n88nn89luA0AJWb9i8Xq9ampqUkdHhzuWy+XU0dGhSCRi+3AAytGNPP0ZT1tbm/H5fGb//v3m9OnT5oknnjA1NTUmlUpN+FrHcUp+15uiqPHLcZwJf4+tvxWSpLVr1+pf//qXduzYoVQqpU996lP685//fNUNXQC3p5vyB3I3IpvNKhAIlLoNAONwHEd+v/+ac0r+VAjA7YdgAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsK7oYHn11Vf1xS9+UfX19aqoqNCLL76Yt90Yox07dmjOnDmaOnWqotGozp49a6tfAJNA0cFy6dIl3XvvvdqzZ8+Y25955hnt3r1be/fuVVdXl6ZPn66VK1dqaGjohpsFMElMuGz8NUgyBw4ccH/O5XImFAqZZ5991h3LZDLG5/OZF154oaB9Oo5T8Kr3FEXd+nIcZ8LfY6v3WM6fP69UKqVoNOqOBQIBNTc3K5FIjPma4eFhZbPZvAIwuVkNllQqJUkKBoN548Fg0N32QfF4XIFAwK2GhgabLQEogZI/FWptbZXjOG4lk8lStwTgBlkNllAoJElKp9N54+l02t32QT6fT36/P68ATG5Wg2XBggUKhULq6Ohwx7LZrLq6uhSJRGweCkAZqyr2BRcvXtS5c+fcn8+fP68TJ05o5syZCofD2rp1q7773e9q4cKFWrBggbZv3676+nqtWrXKZt8Aylmxj5hfeeWVMR9BrVu3zn3kvH37dhMMBo3P5zMrVqwwvb29Be+fx80UVd5VyOPmCmOMURnJZrMKBAKlbgPAOBzHmfBeaMmfCgG4/RAsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hUVLPF4XA888IBmzJih2bNna9WqVert7c2bMzQ0pFgsprq6OlVXV6ulpUXpdNpq0wDKW1HB0tnZqVgspjfeeEOHDx/W6OioPv/5z+vSpUvunG3btungwYNqb29XZ2en+vv7tWbNGuuNAyhjxS4K/34XLlwwkkxnZ6cxxphMJmM8Ho9pb29355w5c8ZIMolEoqB9sig8RZV3FbIo/A3dY3EcR5I0c+ZMSVJPT49GR0cVjUbdOY2NjQqHw0okEmPuY3h4WNlsNq8ATG7XHSy5XE5bt27V8uXLtWjRIklSKpWS1+tVTU1N3txgMKhUKjXmfuLxuAKBgFsNDQ3X2xKAMnHdwRKLxXTq1Cm1tbXdUAOtra1yHMetZDJ5Q/sDUHpV1/OiTZs26aWXXtKrr76quXPnuuOhUEgjIyPKZDJ5Vy3pdFqhUGjMffl8Pvl8vutpA0C5KuZmbS6XM7FYzNTX15u33377qu3/u3n7hz/8wR37+9//biRu3lLU7VKF3LwtKlg2bNhgAoGAOXr0qBkYGHDrvffec+esX7/ehMNhc+TIEdPd3W0ikYiJRCIFH4NgoajyLuvBMt6B9u3b5865fPmy2bhxo6mtrTXTpk0zq1evNgMDAwQLRd0mVUiwVPx/YJSNbDarQCBQ6jYAjMNxHPn9/mvO4bNCAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwrKliee+45LV68WH6/X36/X5FIRC+//LK7fWhoSLFYTHV1daqurlZLS4vS6bT1pgGUt6KCZe7cudq1a5d6enrU3d2tz33uc3rkkUf01ltvSZK2bdumgwcPqr29XZ2dnerv79eaNWtuSuMAylgxi8KPpba21vzyl780mUzGeDwe097e7m47c+aMkWQSiUTB+2NReIoq7ypkUfjrvsdy5coVtbW16dKlS4pEIurp6dHo6Kii0ag7p7GxUeFwWIlEYtz9DA8PK5vN5hWAya3oYDl58qSqq6vl8/m0fv16HThwQPfcc49SqZS8Xq9qamry5geDQaVSqXH3F4/HFQgE3GpoaCj6HwGgvBQdLHfffbdOnDihrq4ubdiwQevWrdPp06evu4HW1lY5juNWMpm87n0BKA9Vxb7A6/Xqox/9qCSpqalJx48f149//GOtXbtWIyMjymQyeVct6XRaoVBo3P35fD75fL7iOwdQtm7471hyuZyGh4fV1NQkj8ejjo4Od1tvb6/6+voUiURu9DAAJpGirlhaW1v10EMPKRwOa3BwUL/97W919OhRHTp0SIFAQI8//riefPJJzZw5U36/X5s3b1YkEtGyZctuVv8AylExj5a/8Y1vmHnz5hmv12s+9KEPmRUrVpi//OUv7vbLly+bjRs3mtraWjNt2jSzevVqMzAwUMwheNxMUWVehTxurjDGGJWRbDarQCBQ6jYAjMNxHPn9/mvO4bNCAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKy7oWDZtWuXKioqtHXrVndsaGhIsVhMdXV1qq6uVktLi9Lp9I32CWASue5gOX78uH72s59p8eLFeePbtm3TwYMH1d7ers7OTvX392vNmjU33CiASaSoFdv/3+DgoFm4cKE5fPiwefDBB82WLVuMMcZkMhnj8XhMe3u7O/fMmTNGkkkkEgXtm0XhKaq8q5BF4a/riiUWi+nhhx9WNBrNG+/p6dHo6GjeeGNjo8LhsBKJxJj7Gh4eVjabzSsAk1tVsS9oa2vTm2++qePHj1+1LZVKyev1qqamJm88GAwqlUqNub94PK6dO3cW2waAMlbUFUsymdSWLVv0/PPPa8qUKVYaaG1tleM4biWTSSv7BVA6RQVLT0+PLly4oPvvv19VVVWqqqpSZ2endu/eraqqKgWDQY2MjCiTyeS9Lp1OKxQKjblPn88nv9+fVwAmt6LeCq1YsUInT57MG/v617+uxsZGPfXUU2poaJDH41FHR4daWlokSb29verr61MkErHXNYCyVlSwzJgxQ4sWLcobmz59uurq6tzxxx9/XE8++aRmzpwpv9+vzZs3KxKJaNmyZfa6BlDWir55O5Ef/vCHqqysVEtLi4aHh7Vy5Ur99Kc/tX0YAGWswhhjSt3E+2WzWQUCgVK3AWAcjuNMeC+UzwoBsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsK6oYPnOd76jioqKvGpsbHS3Dw0NKRaLqa6uTtXV1WppaVE6nbbeNIDyVvQVyyc+8QkNDAy49dprr7nbtm3bpoMHD6q9vV2dnZ3q7+/XmjVrrDYMYBIwRXj66afNvffeO+a2TCZjPB6PaW9vd8fOnDljJJlEIlHwMRzHMZIoiirTchxnwt/joq9Yzp49q/r6en3kIx/RY489pr6+PklST0+PRkdHFY1G3bmNjY0Kh8NKJBLFHgbAJFZVzOTm5mbt379fd999twYGBrRz50599rOf1alTp5RKpeT1elVTU5P3mmAwqFQqNe4+h4eHNTw87P6czWaL+xcAKDtFBctDDz3k/vfixYvV3NysefPm6fe//72mTp16XQ3E43Ht3Lnzul4LoDzd0OPmmpoafexjH9O5c+cUCoU0MjKiTCaTNyedTisUCo27j9bWVjmO41YymbyRlgCUgRsKlosXL+of//iH5syZo6amJnk8HnV0dLjbe3t71dfXp0gkMu4+fD6f/H5/XgGY5Ap+XGOM+fa3v22OHj1qzp8/b15//XUTjUbNrFmzzIULF4wxxqxfv96Ew2Fz5MgR093dbSKRiIlEIsUcgqdCFFXmVchToaKCZe3atWbOnDnG6/WaD3/4w2bt2rXm3Llz7vbLly+bjRs3mtraWjNt2jSzevVqMzAwQLBQ1G1UhQRLhTHGqIxks1kFAoFStwFgHI7jTHjLgs8KAbCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1Ra2EiNtbmX2velEqKipK3QLehysWANYRLACsI1gAWEewALCOYAFgXdHB8s9//lNf+cpXVFdXp6lTp+qTn/ykuru73e3GGO3YsUNz5szR1KlTFY1GdfbsWatNAyhvRQXLf/7zHy1fvlwej0cvv/yyTp8+rR/84Aeqra115zzzzDPavXu39u7dq66uLk2fPl0rV67U0NCQ9eYBlKkJl41/n6eeesp85jOfGXd7LpczoVDIPPvss+5YJpMxPp/PvPDCCwUdw3Gcgle9p+zWZFbqc3cnleM4E/7/KOqK5U9/+pOWLFmiRx99VLNnz9Z9992nX/ziF+728+fPK5VKKRqNumOBQEDNzc1KJBJj7nN4eFjZbDavAExuRQXLO++8o+eee04LFy7UoUOHtGHDBn3rW9/Sr3/9a0lSKpWSJAWDwbzXBYNBd9sHxeNxBQIBtxoaGq7n3wGgnBRzuenxeEwkEskb27x5s1m2bJkxxpjXX3/dSDL9/f15cx599FHz5S9/ecx9Dg0NGcdx3EomkyW/1KMoavyy/lZozpw5uueee/LGPv7xj6uvr0+SFAqFJEnpdDpvTjqddrd9kM/nk9/vzysAk1tRwbJ8+XL19vbmjb399tuaN2+eJGnBggUKhULq6Ohwt2ezWXV1dSkSiVhoF8CkUMxboWPHjpmqqirzve99z5w9e9Y8//zzZtq0aeY3v/mNO2fXrl2mpqbG/PGPfzR/+9vfzCOPPGIWLFhgLl++XNAxeCpEUeVdhbwVKvoZ48GDB82iRYuMz+czjY2N5uc//3ne9lwuZ7Zv326CwaDx+XxmxYoVpre3t+D9EywUVd5VSLBUGFNeX8KRzWYVCARK3QaAcTiOM+G9UD4rBMA6ggWAdQQLAOsIFgDWlV2wlNm9ZAAfUMjvaNkFy+DgYKlbAHANhfyOlt3j5lwup/7+fhljFA6HlUwm+TP/MWSzWTU0NHB+roFzdG3Fnh9jjAYHB1VfX6/Kymtfk5TdukKVlZWaO3eu+/UJfH7o2jg/E+McXVsx56fQvzEru7dCACY/ggWAdWUbLD6fT08//bR8Pl+pWylLnJ+JcY6u7Waen7K7eQtg8ivbKxYAkxfBAsA6ggWAdQQLAOvKNlj27Nmj+fPna8qUKWpubtaxY8dK3VJJxONxPfDAA5oxY4Zmz56tVatWXfW9w0NDQ4rFYqqrq1N1dbVaWlqu+kLzO8GuXbtUUVGhrVu3umOcmxIti1zwd0beQm1tbcbr9Zpf/epX5q233jLf/OY3TU1NjUmn06Vu7ZZbuXKl2bdvnzl16pQ5ceKE+cIXvmDC4bC5ePGiO2f9+vWmoaHBdHR0mO7ubrNs2TLz6U9/uoRd33rHjh0z8+fPN4sXLzZbtmxxx+/0c/Pvf//bzJs3z3zta18zXV1d5p133jGHDh0y586dc+fs2rXLBAIB8+KLL5q//vWv5ktf+lJR31M9lrIMlqVLl5pYLOb+fOXKFVNfX2/i8XgJuyoPFy5cMJJMZ2enMea/S9h6PB7T3t7uzjlz5oyRZBKJRKnavKUGBwfNwoULzeHDh82DDz7oBgvn5tYsizyWsnsrNDIyop6enrxlWisrKxWNRsddpvVO4jiOJGnmzJmSpJ6eHo2Ojuadr8bGRoXD4TvmfMViMT388MN550Di3Eg3Z1nkQpRdsLz77ru6cuVKUcu03ilyuZy2bt2q5cuXa9GiRZL+u6yt1+tVTU1N3tw75Xy1tbXpzTffVDwev2rbnX5upJuzLHIhyu7TzRhfLBbTqVOn9Nprr5W6lbKQTCa1ZcsWHT58WFOmTCl1O2Upl8tpyZIl+v73vy9Juu+++3Tq1Cnt3btX69atu2nHLbsrllmzZumuu+4qapnWO8GmTZv00ksv6ZVXXtHcuXPd8VAopJGREWUymbz5d8L56unp0YULF3T//ferqqpKVVVV6uzs1O7du1VVVaVgMHjHnpv/uRnLIhei7ILF6/Wqqakpb5nWXC6njo6OO3KZVmOMNm3apAMHDujIkSNasGBB3vampiZ5PJ6889Xb26u+vr7b/nytWLFCJ0+e1IkTJ9xasmSJHnvsMfe/79Rz8z8lWxb5um/73kRtbW3G5/OZ/fv3m9OnT5snnnjC1NTUmFQqVerWbrkNGzaYQCBgjh49agYGBtx677333Dnr16834XDYHDlyxHR3d5tIJGIikUgJuy6d9z8VMoZzcyuWRR5LWQaLMcb85Cc/MeFw2Hi9XrN06VLzxhtvlLqlktA4y1zu27fPnXP58mWzceNGU1tba6ZNm2ZWr15tBgYGStd0CX0wWDg3N39Z5LHwtQkArCu7eywAJj+CBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANb9H7zU2aMyPyeJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f64b79c2760>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-6.9, 1.3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/1.PPO.Catcher')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11.05, 0.9733961166965893)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/1.PPO.Catcher')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEWCAYAAACjTbhPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVfUlEQVR4nO3df2xVd/3H8Ve73nv5Ue5tKXIvlV7AiKsTmVsZ5YpmidxI5qIDmskfM6IuLsAFgfnH7B8wSdTbbDEqBoc/Ipg4V60JU5ZMJGV02XJXaBcUBnbgSHpjey8ues8tjP4I9/P9w68n3tHS3vKBewvPR/JO1s/53HPenKyvnHtO7/2UGWOMAMCi8mI3AOD2Q7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsO6mBcvevXu1cOFCTZs2TY2NjTp+/PjNOhSAElN2Mz4r9Nvf/lZf/vKXtW/fPjU2NuqHP/yh2tra1NPTo7lz5173tblcTn19fZo1a5bKyspstwZgkowxGhgYUG1trcrLx7kmMTfB8uXLTSwWc3++evWqqa2tNfF4fNzXJpNJI4miqBKtZDI57u+x9bdCw8PD6u7uVjQadcfKy8sVjUaVSCSumT80NKRsNuuW4cPWQEmbNWvWuHOsB8u7776rq1evKhgM5o0Hg0GlUqlr5sfjcQUCAbfC4bDtlgBYNJFbFEV/KtTc3CzHcdxKJpPFbgnADaqwvcM5c+borrvuUjqdzhtPp9MKhULXzPf5fPL5fLbbAFBE1q9YvF6vGhoa1N7e7o7lcjm1t7crEonYPhyAUnQjT3/G0traanw+nzlw4IA5c+aMeeKJJ0xVVZVJpVLjvtZxnKLf9aYoauxyHGfc32Prb4Ukaf369frnP/+pXbt2KZVK6ROf+IT+9Kc/XXNDF8Dt6ab8gdyNyGazCgQCxW4DwBgcx5Hf77/unKI/FQJw+yFYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCs4WF599VV9/vOfV21trcrKyvTiiy/mbTfGaNeuXZo3b56mT5+uaDSqc+fO2eoXwBRQcLBcvnxZ9957r/bu3Tvq9meeeUZ79uzRvn371NnZqZkzZ2r16tUaHBy84WYBTBHjLht/HZLMwYMH3Z9zuZwJhULm2WefdccymYzx+XzmhRdemNA+HceZ8Kr3FEXd+nIcZ9zfY6v3WC5cuKBUKqVoNOqOBQIBNTY2KpFIjPqaoaEhZbPZvAIwtVkNllQqJUkKBoN548Fg0N32fvF4XIFAwK26ujqbLQEogqI/FWpubpbjOG4lk8litwTgBlkNllAoJElKp9N54+l02t32fj6fT36/P68ATG1Wg2XRokUKhUJqb293x7LZrDo7OxWJRGweCkAJqyj0BZcuXdL58+fdny9cuKCTJ09q9uzZCofD2r59u77zne9o8eLFWrRokXbu3Kna2lqtWbPGZt8ASlmhj5hfeeWVUR9BbdiwwX3kvHPnThMMBo3P5zOrVq0yPT09E94/j5spqrRrIo+by4wxRiUkm80qEAgUuw0AY3AcZ9x7oUV/KgTg9kOwALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWFdQsMTjcT3wwAOaNWuW5s6dqzVr1qinpydvzuDgoGKxmGpqalRZWammpial02mrTQMobQUFS0dHh2KxmN544w0dOXJEIyMj+uxnP6vLly+7c3bs2KFDhw6pra1NHR0d6uvr07p166w3DqCEFboo/P+6ePGikWQ6OjqMMcZkMhnj8XhMW1ubO+fs2bNGkkkkEhPaJ4vCU1Rp10QWhb+heyyO40iSZs+eLUnq7u7WyMiIotGoO6e+vl7hcFiJRGLUfQwNDSmbzeYVgKlt0sGSy+W0fft2rVy5UkuWLJEkpVIpeb1eVVVV5c0NBoNKpVKj7icejysQCLhVV1c32ZYAlIhJB0ssFtPp06fV2tp6Qw00NzfLcRy3ksnkDe0PQPFVTOZFW7Zs0UsvvaRXX31V8+fPd8dDoZCGh4eVyWTyrlrS6bRCodCo+/L5fPL5fJNpA0CpKuRmbS6XM7FYzNTW1pq33377mu3/vXn7+9//3h3729/+ZiRu3lLU7VITuXlbULBs2rTJBAIBc+zYMdPf3+/We++9587ZuHGjCYfD5ujRo6arq8tEIhETiUQmfAyChaJKu6wHy1gH2r9/vzvnypUrZvPmzaa6utrMmDHDrF271vT39xMsFHWb1ESCpez/A6NkZLNZBQKBYrcBYAyO48jv9193Dp8VAmAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWBdQcHy3HPPaenSpfL7/fL7/YpEInr55Zfd7YODg4rFYqqpqVFlZaWampqUTqetNw2gtBUULPPnz1dLS4u6u7vV1dWlz3zmM3rkkUf01ltvSZJ27NihQ4cOqa2tTR0dHerr69O6detuSuMASlghi8KPprq62vziF78wmUzGeDwe09bW5m47e/askWQSicSE98ei8BRV2jWRReEnfY/l6tWram1t1eXLlxWJRNTd3a2RkRFFo1F3Tn19vcLhsBKJxJj7GRoaUjabzSsAU1vBwXLq1ClVVlbK5/Np48aNOnjwoO655x6lUil5vV5VVVXlzQ8Gg0qlUmPuLx6PKxAIuFVXV1fwPwJAaSk4WO6++26dPHlSnZ2d2rRpkzZs2KAzZ85MuoHm5mY5juNWMpmc9L4AlIaKQl/g9Xr14Q9/WJLU0NCgEydO6Ec/+pHWr1+v4eFhZTKZvKuWdDqtUCg05v58Pp98Pl/hnQMoWTf8dyy5XE5DQ0NqaGiQx+NRe3u7u62np0e9vb2KRCI3ehgAU0hBVyzNzc166KGHFA6HNTAwoN/85jc6duyYDh8+rEAgoMcff1xPPvmkZs+eLb/fr61btyoSiWjFihU3q38ApaiQR8tf+9rXzIIFC4zX6zUf+MAHzKpVq8yf//xnd/uVK1fM5s2bTXV1tZkxY4ZZu3at6e/vL+QQPG6mqBKviTxuLjPGGJWQbDarQCBQ7DYAjMFxHPn9/uvO4bNCAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKy7oWBpaWlRWVmZtm/f7o4NDg4qFouppqZGlZWVampqUjqdvtE+AUwhkw6WEydO6Kc//amWLl2aN75jxw4dOnRIbW1t6ujoUF9fn9atW3fDjQKYQgpasf3/DQwMmMWLF5sjR46YBx980Gzbts0YY0wmkzEej8e0tbW5c8+ePWskmUQiMaF9syg8RZV2TWRR+EldscRiMT388MOKRqN5493d3RoZGckbr6+vVzgcViKRGHVfQ0NDymazeQVgaqso9AWtra168803deLEiWu2pVIpeb1eVVVV5Y0Hg0GlUqlR9xePx7V79+5C2wBQwgq6Ykkmk9q2bZuef/55TZs2zUoDzc3NchzHrWQyaWW/AIqnoGDp7u7WxYsXdf/996uiokIVFRXq6OjQnj17VFFRoWAwqOHhYWUymbzXpdNphUKhUffp8/nk9/vzCsDUVtBboVWrVunUqVN5Y1/96ldVX1+vp556SnV1dfJ4PGpvb1dTU5MkqaenR729vYpEIva6BlDSCgqWWbNmacmSJXljM2fOVE1NjTv++OOP68knn9Ts2bPl9/u1detWRSIRrVixwl7XAEpawTdvx/ODH/xA5eXlampq0tDQkFavXq2f/OQntg8DoISVGWNMsZv4X9lsVoFAoNhtABiD4zjj3gvls0IArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCsoWL797W+rrKwsr+rr693tg4ODisViqqmpUWVlpZqampROp603DaC0FXzF8rGPfUz9/f1uvfbaa+62HTt26NChQ2pra1NHR4f6+vq0bt06qw0DmAJMAZ5++mlz7733jrotk8kYj8dj2tra3LGzZ88aSSaRSEz4GI7jGEkURZVoOY4z7u9xwVcs586dU21trT70oQ/pscceU29vrySpu7tbIyMjikaj7tz6+nqFw2ElEolCDwNgCqsoZHJjY6MOHDigu+++W/39/dq9e7c+/elP6/Tp00qlUvJ6vaqqqsp7TTAYVCqVGnOfQ0NDGhoacn/OZrOF/QsAlJyCguWhhx5y/3vp0qVqbGzUggUL9Lvf/U7Tp0+fVAPxeFy7d++e1GsBlKYbetxcVVWlj3zkIzp//rxCoZCGh4eVyWTy5qTTaYVCoTH30dzcLMdx3EomkzfSEoAScEPBcunSJf3973/XvHnz1NDQII/Ho/b2dnd7T0+Pent7FYlExtyHz+eT3+/PKwBT3IQf1xhjvvnNb5pjx46ZCxcumNdff91Eo1EzZ84cc/HiRWOMMRs3bjThcNgcPXrUdHV1mUgkYiKRSCGH4KkQRZV4TeSpUEHBsn79ejNv3jzj9XrNBz/4QbN+/Xpz/vx5d/uVK1fM5s2bTXV1tZkxY4ZZu3at6e/vJ1go6jaqiQRLmTHGqIRks1kFAoFitwFgDI7jjHvLgs8KAbCOYAFgHcECwDqCBYB1BAsA6wgWANYV9FkhXMt861uTfm1ZS4vFToDSwRULAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACw7rb+Mu0S+6cVpKysrNgtAKPiy7QBFAXBAsA6ggWAdQQLAOsIFgDWFRws//jHP/SlL31JNTU1mj59uj7+8Y+rq6vL3W6M0a5duzRv3jxNnz5d0WhU586ds9o0gNJWULD8+9//1sqVK+XxePTyyy/rzJkz+v73v6/q6mp3zjPPPKM9e/Zo37596uzs1MyZM7V69WoNDg5abx5AiRp32fj/8dRTT5lPfepTY27P5XImFAqZZ5991h3LZDLG5/OZF154YULHcBxnwqvej1dTma1zQFG2y3Gccf//LeiK5Y9//KOWLVumRx99VHPnztV9992nn//85+72CxcuKJVKKRqNumOBQECNjY1KJBKj7nNoaEjZbDavAExtBQXLO++8o+eee06LFy/W4cOHtWnTJn3jG9/Qr371K0lSKpWSJAWDwbzXBYNBd9v7xeNxBQIBt+rq6ibz7wBQSgq5PPd4PCYSieSNbd261axYscIYY8zrr79uJJm+vr68OY8++qj54he/OOo+BwcHjeM4biWTyaJf6lEUNXZZfys0b9483XPPPXljH/3oR9Xb2ytJCoVCkqR0Op03J51Ou9vez+fzye/35xWAqa2gYFm5cqV6enryxt5++20tWLBAkrRo0SKFQiG1t7e727PZrDo7OxWJRCy0C2BKKOSt0PHjx01FRYX57ne/a86dO2eef/55M2PGDPPrX//andPS0mKqqqrMH/7wB/PXv/7VPPLII2bRokXmypUrEzqGzadCFEXZr4m8FSr4meyhQ4fMkiVLjM/nM/X19eZnP/tZ3vZcLmd27txpgsGg8fl8ZtWqVaanp2fC+ydYKKq0ayLBclt/HwsA+/g+FgBFQbAAsI5gAWAdwQLAupILlhK7lwzgfSbyO1pywTIwMFDsFgBcx0R+R0vucXMul1NfX5+MMQqHw0omk/yZ/yiy2azq6uo4P9fBObq+Qs+PMUYDAwOqra1Vefn1r0kqbDVpS3l5uebPn+9+fQKfH7o+zs/4OEfXV8j5mejfmJXcWyEAUx/BAsC6kg0Wn8+np59+Wj6fr9itlCTOz/g4R9d3M89Pyd28BTD1lewVC4Cpi2ABYB3BAsA6ggWAdSUbLHv37tXChQs1bdo0NTY26vjx48VuqSji8bgeeOABzZo1S3PnztWaNWuu+d7hwcFBxWIx1dTUqLKyUk1NTdd8ofmdoKWlRWVlZdq+fbs7xrkp0rLIE/7OyFuotbXVeL1e88tf/tK89dZb5utf/7qpqqoy6XS62K3dcqtXrzb79+83p0+fNidPnjSf+9znTDgcNpcuXXLnbNy40dTV1Zn29nbT1dVlVqxYYT75yU8Wsetb7/jx42bhwoVm6dKlZtu2be74nX5u/vWvf5kFCxaYr3zlK6azs9O888475vDhw+b8+fPunJaWFhMIBMyLL75o/vKXv5gvfOELBX1P9WhKMliWL19uYrGY+/PVq1dNbW2ticfjReyqNFy8eNFIMh0dHcaY/yxh6/F4TFtbmzvn7NmzRpJJJBLFavOWGhgYMIsXLzZHjhwxDz74oBssnJtbsyzyaErurdDw8LC6u7vzlmktLy9XNBodc5nWO4njOJKk2bNnS5K6u7s1MjKSd77q6+sVDofvmPMVi8X08MMP550DiXMj3ZxlkSei5ILl3Xff1dWrVwtapvVOkcvltH37dq1cuVJLliyR9J9lbb1er6qqqvLm3innq7W1VW+++abi8fg12+70cyPdnGWRJ6LkPt2MscViMZ0+fVqvvfZasVspCclkUtu2bdORI0c0bdq0YrdTknK5nJYtW6bvfe97kqT77rtPp0+f1r59+7Rhw4abdtySu2KZM2eO7rrrroKWab0TbNmyRS+99JJeeeUVzZ8/3x0PhUIaHh5WJpPJm38nnK/u7m5dvHhR999/vyoqKlRRUaGOjg7t2bNHFRUVCgaDd+y5+a+bsSzyRJRcsHi9XjU0NOQt05rL5dTe3n5HLtNqjNGWLVt08OBBHT16VIsWLcrb3tDQII/Hk3e+enp61Nvbe9ufr1WrVunUqVM6efKkW8uWLdNjjz3m/vedem7+q2jLIk/6tu9N1Nraanw+nzlw4IA5c+aMeeKJJ0xVVZVJpVLFbu2W27RpkwkEAubYsWOmv7/frffee8+ds3HjRhMOh83Ro0dNV1eXiUQiJhKJFLHr4vnfp0LGcG5uxbLIoynJYDHGmB//+McmHA4br9drli9fbt54441it1QUGmOZy/3797tzrly5YjZv3myqq6vNjBkzzNq1a01/f3/xmi6i9wcL5+bmL4s8Gr42AYB1JXePBcDUR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAuv8D7j7mnqe2x7UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.0 400 [0.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 3 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
